<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml">
  <head>

    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
    <meta content="Cask Data, Inc." name="author" />
<meta content="Copyright © 2015-2016 Cask Data, Inc." name="copyright" />


    <meta name="git_release" content="6.1.1">
    <meta name="git_hash" content="05fbac36f9f7aadeb44f5728cea35136dbc243e5">
    <meta name="git_timestamp" content="2020-02-09 08:22:47 +0800">
    <title>Data Analysis with OLAP Cube</title>

    <link rel="stylesheet" href="../_static/cdap-bootstrap.css" type="text/css" />
    <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
    <link rel="stylesheet" href="../_static/bootstrap-3.3.6/css/bootstrap.min.css" type="text/css" />
    <link rel="stylesheet" href="../_static/bootstrap-3.3.6/css/bootstrap-theme.min.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/bootstrap-sphinx.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/cdap-dynamicscrollspy-4.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/jquery.mCustomScrollbar.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/cdap-jquery.mCustomScrollbar.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/abixTreeList-2.css" type="text/css" />
    <link rel="stylesheet" href="../_static/cdap-bootstrap.css" type="text/css" />

    <script type="text/javascript">
      var DOCUMENTATION_OPTIONS = {
        URL_ROOT:    '',
        VERSION:     '6.1.1',
        COLLAPSE_INDEX: false,
        FILE_SUFFIX: '.html',
        HAS_SOURCE:  false
      };
    </script>
    <script type="text/javascript" src="../_static/jquery.js"></script>
    <script type="text/javascript" src="../_static/underscore.js"></script>
    <script type="text/javascript" src="../_static/doctools.js"></script>
    <script type="text/javascript" src="../_static/language_data.js"></script>

    <link rel="shortcut icon" href="../_static/favicon.ico"/>
    <link rel="index" title="Index" href="../genindex.html" />
    <link rel="search" title="Search" href="../search.html" />
    <link rel="top" title="Cask Data Application Platform 6.1.1 Documentation" href="../index.html" />
    <link rel="up" title="How-To Guides" href="index.html" />
    <link rel="next" title="Creating Hydrator Applications using CDAP System Artifacts" href="cdap-etl-guide.html" />
    <link rel="prev" title="Analyzing CDAP Data from BI Tools" href="cdap-bi-guide.html" />
    <!-- block extrahead -->
    <meta charset='utf-8'>
    <meta http-equiv='X-UA-Compatible' content='IE=edge,chrome=1'>
    <meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1'>
    <meta name="apple-mobile-web-app-capable" content="yes">
    <!-- block extrahead end -->

</head>
<body role="document">

<!-- block navbar -->
<div id="navbar" class="navbar navbar-inverse navbar-default navbar-fixed-top">
    <div class="container-fluid">
      <div class="row">
        <div class="navbar-header">
          <!-- .btn-navbar is used as the toggle for collapsed navbar content -->
          <a class="navbar-brand" href="../table-of-contents/../../index.html">
            <span><img alt="CDAP logo" src="../_static/cdap_logo.svg"/></span>
          </a>

          <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".nav-collapse">
            <span class="icon-bar"></span>
            <span class="icon-bar"></span>
            <span class="icon-bar"></span>
          </button>

          <div class="pull-right">
            <div class="dropdown version-dropdown">
              <a href="#" class="dropdown-toggle" data-toggle="dropdown"
                role="button" aria-haspopup="true" aria-expanded="false">
                v 6.1.1 <span class="caret"></span>
              </a>
              <ul class="dropdown-menu">
                <li><a href="//docs.cdap.io/cdap/5.1.2/en/index.html">v 5.1.2</a></li>
                <li><a href="//docs.cdap.io/cdap/4.3.4/en/index.html">v 4.3.4</a></li>
              </ul>
            </div>
          </div>
          <form class="navbar-form navbar-right navbar-search" action="../search.html" method="get">
            <div class="form-group">
              <div class="navbar-search-image material-icons"></div>
              <input type="text" name="q" class="form-control" placeholder="  Search" />
            </div>
            <input type="hidden" name="check_keywords" value="yes" />
            <input type="hidden" name="area" value="default" />
          </form>

          <div class="collapse navbar-collapse nav-collapse navbar-right navbar-navigation">
            <ul class="nav navbar-nav"><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../table-of-contents/../../index.html">简介</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link current" href="../table-of-contents/../../guides.html">手册</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../table-of-contents/../../reference-manual/index.html">参考</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../table-of-contents/../../faqs/index.html">帮助</a></li>
            </ul>
          </div>

        </div>
      </div>
    </div>
  </div><!-- block navbar end -->
<!-- block main content -->
<div class="main-container container">
  <div class="row"><div class="col-md-2">
      <div id="sidebar" class="bs-sidenav scrollable-y-outside" role="complementary">
<!-- theme_manual: examples-manual -->
<!-- theme_manual_highlight: guides -->
<!-- sidebar_title_link: ../table-of-contents/../../guides.html -->

  <div role="note" aria-label="manuals links"><h3><a href="../table-of-contents/../../guides.html">Guides</a></h3>

    <ul class="this-page-menu">
      <li class="toctree-l1"><a href="../table-of-contents/../../user-guide/index.html" rel="nofollow">用户手册</a>
      </li>
      <li class="toctree-l1"><a href="../table-of-contents/../../developer-manual/index.html" rel="nofollow">开发手册</a>
      </li>
      <li class="toctree-l1"><a href="../table-of-contents/../../admin-manual/index.html" rel="nofollow">管理手册</a>
      </li>
      <li class="toctree-l1"><a href="../table-of-contents/../../integrations/index.html" rel="nofollow">集成手册</a>
      </li>
      <li class="toctree-l1"><b><a href="../table-of-contents/../../examples-manual/index.html" rel="nofollow">最佳实践</a></b>
      <nav class="pagenav">
      <ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../index.html">Introduction</a></li>
<li class="toctree-l1 current"><a class="reference internal" href="index.html">How-To Guides</a><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="cdap-bi-guide.html">Analyzing CDAP Data from BI Tools</a></li>
<li class="toctree-l2 current"><a class="current reference internal" href="#">Data Analysis with OLAP Cube</a></li>
<li class="toctree-l2"><a class="reference internal" href="cdap-etl-guide.html">Creating Hydrator Applications using CDAP System Artifacts</a></li>
<li class="toctree-l2"><a class="reference internal" href="cdap-flow-guide.html">Real-Time Data Processing with a Flow</a></li>
<li class="toctree-l2"><a class="reference internal" href="cdap-flume-guide.html">Ingesting Data into CDAP using Apache Flume</a></li>
<li class="toctree-l2"><a class="reference internal" href="cdap-kafka-ingest-guide.html">Consuming Data from Kafka</a></li>
<li class="toctree-l2"><a class="reference internal" href="cdap-mapreduce-guide.html">Batch Data Processing with CDAP</a></li>
<li class="toctree-l2"><a class="reference internal" href="cdap-spark-guide.html">Iterative Data Processing with Apache Spark</a></li>
<li class="toctree-l2"><a class="reference internal" href="cdap-timeseries-guide.html">Storing Timeseries Data</a></li>
<li class="toctree-l2"><a class="reference internal" href="cdap-twitter-ingest-guide.html">Consuming Twitter Data in Real Time</a></li>
<li class="toctree-l2"><a class="reference internal" href="cdap-workflow-guide.html">Batch Data Processing with CDAP using Workflow</a></li>
</ul>
</li>
</ul>
</nav>
      </li>
    </ul>
  </div></div>
    </div><div class="col-md-8 content" id="main-content">
    
  <div class="section" id="data-analysis-with-olap-cube">
<span id="cdap-cube-guide"></span><h1>Data Analysis with OLAP Cube<a class="headerlink" href="#data-analysis-with-olap-cube" title="Permalink to this headline">🔗</a></h1>
<blockquote class="pull-quote">
<div><strong>Source Code Repository:</strong> Source code (and other resources) for this guide are available at the
<a class="reference external" href="https://github.com/cdap-guides/cdap-cube-guide/tree/develop">CDAP Guides GitHub repository</a>.</div></blockquote>
<p>The <a class="reference external" href="http://cdap.io">Cask Data Application Platform (CDAP)</a> provides a number of
pre-packaged Datasets, which make it easy to store and retrieve data using
best-practices-based implementations of common data access patterns. In this guide, you
will learn how to store multidimensional data points in a Cube dataset
(see <a class="reference external" href="http://en.wikipedia.org/wiki/OLAP_cube">OLAP Cube</a>), and then perform
queries with it. For analysis, we’ll be using an example of processing web logs.</p>
<p>An OLAP (Online Analytical Processing) Cube is multidimensional database or array,
optimized for data warehousing and OLAP applications.</p>
<div class="section" id="what-you-will-build">
<h2>What You Will Build<a class="headerlink" href="#what-you-will-build" title="Permalink to this headline">🔗</a></h2>
<p>This guide will take you through building a simple <a class="reference external" href="http://docs.cdap.io/cdap/current/en/developers-manual/building-blocks/applications.html">CDAP application</a>
that ingests web logs, aggregates the request counts for different combinations of fields,
and that can then be queried for the volume over a time period. You can then retrieve
insights on the traffic of a web site and the web site’s health. You will:</p>
<ul class="simple">
<li>Use a
<a class="reference external" href="http://docs.cdap.io/cdap/current/en/developers-manual/building-blocks/streams.html">Stream</a>
to ingest real-time log data;</li>
<li>Build a
<a class="reference external" href="http://docs.cdap.io/cdap/current/en/developers-manual/building-blocks/flows-flowlets/flows.html">Flow</a>
to process log entries as they are received into multidimensional facts;</li>
<li>Use a
<a class="reference external" href="http://docs.cdap.io/cdap/current/en/developers-manual/building-blocks/datasets/index.html">Dataset</a>
to store the aggregated numbers; and</li>
<li>Build a
<a class="reference external" href="http://docs.cdap.io/cdap/current/en/developers-manual/building-blocks/services.html">Service</a>
to query the aggregated data across multiple dimensions.</li>
</ul>
</div>
<div class="section" id="what-you-will-need">
<h2>What You Will Need<a class="headerlink" href="#what-you-will-need" title="Permalink to this headline">🔗</a></h2>
<ul class="simple">
<li><a class="reference external" href="http://www.oracle.com/technetwork/java/javase/downloads/index.html">JDK 7 or 8</a></li>
<li><a class="reference external" href="http://maven.apache.org/">Apache Maven 3.1+</a></li>
<li><a class="reference external" href="http://docs.cdap.io/cdap/current/en/developers-manual/getting-started/standalone/index.html">CDAP SDK</a></li>
</ul>
</div>
<div class="section" id="lets-build-it">
<h2>Let’s Build It!<a class="headerlink" href="#lets-build-it" title="Permalink to this headline">🔗</a></h2>
<p>The following sections will guide you through building an application from scratch. If you
are interested in deploying and running the application right away, you can clone its
source code from this GitHub repository. In that case, feel free to skip the next two
sections and jump right to the <a class="reference external" href="#build-and-run-application">Build and Run Application</a>
section.</p>
<div class="section" id="application-design">
<h3>Application Design<a class="headerlink" href="#application-design" title="Permalink to this headline">🔗</a></h3>
<p>For this guide we will assume we are processing logs of a web-site that are produced by an
Apache web server. The data could be collected from multiple servers and then sent to our
application over HTTP. There are a number of <a class="reference external" href="http://docs.cdap.io/cdap/current/en/developers-manual/ingesting-tools/index.html">tools</a> that
can help you with the ingestion task. We’ll skip over the details of ingesting the data
(as this is <a class="reference external" href="http://docs.cdap.io/cdap/current/en/examples-manual/index.html">covered elsewhere</a>) and instead focus on
storing and retrieving the data.</p>
<p>The application consists of these components:</p>
<a class="reference internal image-reference" href="../_images/app-design1.png"><img alt="../_images/app-design1.png" class="align-center" src="../_images/app-design1.png" style="width: 8in;" /></a>
<p>Weblogs are sent to a <code class="docutils literal notranslate"><span class="pre">weblogs</span></code> Stream that is consumed by a <code class="docutils literal notranslate"><span class="pre">CubeWriterFlow</span></code>.
The <code class="docutils literal notranslate"><span class="pre">CubeWriterFlow</span></code> has a single <code class="docutils literal notranslate"><span class="pre">CubeWriterFlowlet</span></code> that parses a <code class="docutils literal notranslate"><span class="pre">StreamEvent</span></code>’s
body into a <code class="docutils literal notranslate"><span class="pre">CubeFact</span></code> and writes it into a <code class="docutils literal notranslate"><span class="pre">weblogsCube</span></code> Dataset. The Dataset
is configured to aggregate data for specific combinations of dimensions of a
<code class="docutils literal notranslate"><span class="pre">CubeFact</span></code> and provides a querying interface over the stored aggregations.
The application uses a <code class="docutils literal notranslate"><span class="pre">CubeService</span></code> to provide an HTTP interface for querying
the <code class="docutils literal notranslate"><span class="pre">weblogsCube</span></code>.</p>
</div>
<div class="section" id="implementation">
<h3>Implementation<a class="headerlink" href="#implementation" title="Permalink to this headline">🔗</a></h3>
<p>The first step is to set up our application structure. We will use a standard
Maven project structure for all of the source code files:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">./pom.xml</span>
<span class="go">./src/main/java/co/cask/cdap/guides/cube/CubeHandler.java</span>
<span class="go">./src/main/java/co/cask/cdap/guides/cube/CubeWriterFlow.java</span>
<span class="go">./src/main/java/co/cask/cdap/guides/cube/CubeWriterFlowlet.java</span>
<span class="go">./src/main/java/co/cask/cdap/guides/cube/WebAnalyticsApp.java</span>
</pre></div>
</div>
<p>The application is identified by the <code class="docutils literal notranslate"><span class="pre">WebAnalyticsApp</span></code> class. This class extends
<a class="reference external" href="http://docs.cdap.io/cdap/current/en/reference-manual/javadocs/co/cask/cdap/api/app/AbstractApplication.html">AbstractApplication</a>,
and overrides the <code class="docutils literal notranslate"><span class="pre">configure()</span></code> method to define all of the application components:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="kd">public</span> <span class="kd">class</span> <span class="nc">WebAnalyticsApp</span> <span class="kd">extends</span> <span class="n">AbstractApplication</span> <span class="p">{</span>
  <span class="kd">static</span> <span class="kd">final</span> <span class="n">String</span> <span class="n">APP_NAME</span> <span class="o">=</span> <span class="s">&quot;WebAnalyticsApp&quot;</span><span class="p">;</span>
  <span class="kd">static</span> <span class="kd">final</span> <span class="n">String</span> <span class="n">STREAM_NAME</span> <span class="o">=</span> <span class="s">&quot;weblogs&quot;</span><span class="p">;</span>
  <span class="kd">static</span> <span class="kd">final</span> <span class="n">String</span> <span class="n">CUBE_NAME</span> <span class="o">=</span> <span class="s">&quot;weblogsCube&quot;</span><span class="p">;</span>
  <span class="kd">static</span> <span class="kd">final</span> <span class="n">String</span> <span class="n">SERVICE_NAME</span> <span class="o">=</span> <span class="s">&quot;CubeService&quot;</span><span class="p">;</span>

  <span class="nd">@Override</span>
  <span class="kd">public</span> <span class="kt">void</span> <span class="nf">configure</span><span class="p">()</span> <span class="p">{</span>
    <span class="n">setName</span><span class="p">(</span><span class="n">APP_NAME</span><span class="p">);</span>

    <span class="n">addStream</span><span class="p">(</span><span class="k">new</span> <span class="n">Stream</span><span class="p">(</span><span class="n">STREAM_NAME</span><span class="p">));</span>

    <span class="c1">// configure the Cube dataset</span>
    <span class="n">DatasetProperties</span> <span class="n">props</span> <span class="o">=</span> <span class="n">DatasetProperties</span><span class="p">.</span><span class="na">builder</span><span class="p">()</span>
      <span class="p">.</span><span class="na">add</span><span class="p">(</span><span class="s">&quot;dataset.cube.resolutions&quot;</span><span class="p">,</span> <span class="s">&quot;1,60,3600&quot;</span><span class="p">)</span>
      <span class="p">.</span><span class="na">add</span><span class="p">(</span><span class="s">&quot;dataset.cube.aggregation.agg1.dimensions&quot;</span><span class="p">,</span> <span class="s">&quot;response_status&quot;</span><span class="p">)</span>
      <span class="p">.</span><span class="na">add</span><span class="p">(</span><span class="s">&quot;dataset.cube.aggregation.agg2.dimensions&quot;</span><span class="p">,</span> <span class="s">&quot;ip,browser&quot;</span><span class="p">)</span>
      <span class="p">.</span><span class="na">build</span><span class="p">();</span>
    <span class="n">createDataset</span><span class="p">(</span><span class="n">CUBE_NAME</span><span class="p">,</span> <span class="n">Cube</span><span class="p">.</span><span class="na">class</span><span class="p">,</span> <span class="n">props</span><span class="p">);</span>

    <span class="n">addFlow</span><span class="p">(</span><span class="k">new</span> <span class="n">CubeWriterFlow</span><span class="p">());</span>
    <span class="n">addService</span><span class="p">(</span><span class="n">SERVICE_NAME</span><span class="p">,</span> <span class="k">new</span> <span class="n">CubeHandler</span><span class="p">());</span>
  <span class="p">}</span>
<span class="p">}</span>
</pre></div>
</div>
<p>First, we need a place to receive and process the events. CDAP provides a
<a class="reference external" href="http://docs.cdap.io/cdap/current/en/developers-manual/building-blocks/flows-flowlets/index.html">real-time stream processing system</a>
that is a great match for handling event streams. After first setting
the application name, our <code class="docutils literal notranslate"><span class="pre">WebAnalyticsApp</span></code> adds a new
<a class="reference external" href="http://docs.cdap.io/cdap/current/en/developers-manual/building-blocks/streams.html">Stream</a>.</p>
<p>Then, the application configures a Cube dataset to compute and store
aggregations for combinations of dimensions. Let’s take a closer
look at the properties that are used to configure the Cube dataset:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="p">.</span><span class="na">add</span><span class="p">(</span><span class="s">&quot;dataset.cube.resolutions&quot;</span><span class="p">,</span> <span class="s">&quot;1,60,3600&quot;</span><span class="p">)</span>
<span class="p">.</span><span class="na">add</span><span class="p">(</span><span class="s">&quot;dataset.cube.aggregation.agg1.dimensions&quot;</span><span class="p">,</span> <span class="s">&quot;response_status&quot;</span><span class="p">)</span>
<span class="p">.</span><span class="na">add</span><span class="p">(</span><span class="s">&quot;dataset.cube.aggregation.agg2.dimensions&quot;</span><span class="p">,</span> <span class="s">&quot;ip,browser&quot;</span><span class="p">)</span>
</pre></div>
</div>
<p>A Cube dataset can compute aggregates for multiple time resolutions to provide
a better view of data for both small and large time ranges. For example, you may want to see
data points for each second for the last five minutes, while to build a sensible
chart for a report that covers a week, you may need to see per-hour aggregations.</p>
<p>The code above defines three resolutions: 1 second, 1 minute (60 seconds),
and 1 hour (3600 seconds). When querying the Cube data, you can specify any of
these three depending on your need.</p>
<p>Each aggregation in a Cube is defined by a list of dimensions, which can later be used
for querying. The above code defines two aggregations: “agg1” and agg2”. The first
has only one dimension: <em>response_status</em>. Thus, the Cube will allow queries such as
“number of requests that had a response status 200” or “number of requests for
each response status”.</p>
<p>The second aggregation (“agg2”) defines two dimensions: <em>ip</em> and <em>browser</em>, which allows
querying by ip, by browser, or by using both together, as we shall see below.</p>
<p>After the Cube dataset is configured, the application adds a <code class="docutils literal notranslate"><span class="pre">CubeWriterFlow</span></code> to compute
<code class="docutils literal notranslate"><span class="pre">CubeFact</span></code>s from the <code class="docutils literal notranslate"><span class="pre">StreamEvent</span></code>s and write them to the Cube, and a
<code class="docutils literal notranslate"><span class="pre">CubeService</span></code> that has a single handler that provides an HTTP API to query the Cube.</p>
<p>Let’s take a closer look at these two.</p>
<div class="section" id="cubewriterflow">
<h4>CubeWriterFlow<a class="headerlink" href="#cubewriterflow" title="Permalink to this headline">🔗</a></h4>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="kd">public</span> <span class="kd">class</span> <span class="nc">CubeWriterFlow</span> <span class="kd">extends</span> <span class="n">AbstractFlow</span> <span class="p">{</span>
  <span class="kd">static</span> <span class="kd">final</span> <span class="n">String</span> <span class="n">FLOW_NAME</span> <span class="o">=</span> <span class="s">&quot;CubeWriterFlow&quot;</span><span class="p">;</span>

  <span class="nd">@Override</span>
  <span class="kd">public</span> <span class="kt">void</span> <span class="nf">configure</span><span class="p">()</span> <span class="p">{</span>
    <span class="n">setName</span><span class="p">(</span><span class="n">FLOW_NAME</span><span class="p">);</span>
    <span class="n">setDescription</span><span class="p">(</span><span class="s">&quot;Reads logs from a Stream and writes them to a Cube dataset&quot;</span><span class="p">);</span>
    <span class="n">addFlowlet</span><span class="p">(</span><span class="s">&quot;writer&quot;</span><span class="p">,</span> <span class="k">new</span> <span class="n">CubeWriterFlowlet</span><span class="p">());</span>
    <span class="n">connectStream</span><span class="p">(</span><span class="n">WebAnalyticsApp</span><span class="p">.</span><span class="na">STREAM_NAME</span><span class="p">,</span> <span class="s">&quot;writer&quot;</span><span class="p">);</span>
  <span class="p">}</span>
<span class="p">}</span>
</pre></div>
</div>
<p>The Flow configures a single <code class="docutils literal notranslate"><span class="pre">CubeWriterFlowlet</span></code> to consume data from a Stream:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="kd">public</span> <span class="kd">class</span> <span class="nc">CubeWriterFlowlet</span> <span class="kd">extends</span> <span class="n">AbstractFlowlet</span> <span class="p">{</span>
  <span class="kd">private</span> <span class="kd">static</span> <span class="kd">final</span> <span class="n">Pattern</span> <span class="n">CLF_PATTERN</span> <span class="o">=</span>
    <span class="n">Pattern</span><span class="p">.</span><span class="na">compile</span><span class="p">(</span><span class="s">&quot;^([\\d.]+) (\\S+) (\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] &quot;</span> <span class="o">+</span>
                      <span class="s">&quot;\&quot;(.+?)\&quot; (\\d{3}) (\\d+) \&quot;([^\&quot;]+)\&quot; \&quot;([^\&quot;]+)\&quot;&quot;</span><span class="p">);</span>

  <span class="kd">private</span> <span class="kd">static</span> <span class="kd">final</span> <span class="n">DateFormat</span> <span class="n">DATE_FORMAT</span> <span class="o">=</span> <span class="k">new</span> <span class="n">SimpleDateFormat</span><span class="p">(</span><span class="s">&quot;dd/MMM/yyyy:HH:mm:ss Z&quot;</span><span class="p">);</span>

  <span class="kd">private</span> <span class="n">Metrics</span> <span class="n">metrics</span><span class="p">;</span>

  <span class="nd">@UseDataSet</span><span class="p">(</span><span class="n">WebAnalyticsApp</span><span class="p">.</span><span class="na">CUBE_NAME</span><span class="p">)</span>
  <span class="kd">private</span> <span class="n">Cube</span> <span class="n">cube</span><span class="p">;</span>

  <span class="nd">@ProcessInput</span>
  <span class="kd">public</span> <span class="kt">void</span> <span class="nf">process</span><span class="p">(</span><span class="n">StreamEvent</span> <span class="n">event</span><span class="p">)</span> <span class="kd">throws</span> <span class="n">Exception</span> <span class="p">{</span>

    <span class="n">String</span> <span class="n">logEntryLine</span> <span class="o">=</span> <span class="n">Bytes</span><span class="p">.</span><span class="na">toString</span><span class="p">(</span><span class="n">event</span><span class="p">.</span><span class="na">getBody</span><span class="p">());</span>

    <span class="n">Pattern</span> <span class="n">p</span> <span class="o">=</span> <span class="n">CLF_PATTERN</span><span class="p">;</span>
    <span class="n">Matcher</span> <span class="n">matcher</span> <span class="o">=</span> <span class="n">p</span><span class="p">.</span><span class="na">matcher</span><span class="p">(</span><span class="n">logEntryLine</span><span class="p">);</span>
    <span class="k">if</span> <span class="p">(</span><span class="o">!</span><span class="n">matcher</span><span class="p">.</span><span class="na">matches</span><span class="p">())</span> <span class="p">{</span>
      <span class="n">metrics</span><span class="p">.</span><span class="na">count</span><span class="p">(</span><span class="s">&quot;parse.errors&quot;</span><span class="p">,</span> <span class="mi">1</span><span class="p">);</span>
      <span class="k">return</span><span class="p">;</span>
    <span class="p">}</span>

    <span class="c1">// creating CubeFact with timestamp of the log record</span>
    <span class="kt">long</span> <span class="n">ts</span> <span class="o">=</span> <span class="n">DATE_FORMAT</span><span class="p">.</span><span class="na">parse</span><span class="p">(</span><span class="n">matcher</span><span class="p">.</span><span class="na">group</span><span class="p">(</span><span class="mi">4</span><span class="p">)).</span><span class="na">getTime</span><span class="p">();</span>
    <span class="n">CubeFact</span> <span class="n">fact</span> <span class="o">=</span> <span class="k">new</span> <span class="n">CubeFact</span><span class="p">(</span><span class="n">ts</span> <span class="o">/</span> <span class="mi">1000</span><span class="p">);</span>

    <span class="c1">// adding dimensions</span>
    <span class="n">fact</span><span class="p">.</span><span class="na">addDimensionValue</span><span class="p">(</span><span class="s">&quot;ip&quot;</span><span class="p">,</span> <span class="n">matcher</span><span class="p">.</span><span class="na">group</span><span class="p">(</span><span class="mi">1</span><span class="p">));</span>
    <span class="n">fact</span><span class="p">.</span><span class="na">addDimensionValue</span><span class="p">(</span><span class="s">&quot;request&quot;</span><span class="p">,</span> <span class="n">matcher</span><span class="p">.</span><span class="na">group</span><span class="p">(</span><span class="mi">5</span><span class="p">));</span>
    <span class="n">fact</span><span class="p">.</span><span class="na">addDimensionValue</span><span class="p">(</span><span class="s">&quot;response_status&quot;</span><span class="p">,</span> <span class="n">matcher</span><span class="p">.</span><span class="na">group</span><span class="p">(</span><span class="mi">6</span><span class="p">));</span>
    <span class="k">if</span> <span class="p">(</span><span class="o">!</span><span class="n">matcher</span><span class="p">.</span><span class="na">group</span><span class="p">(</span><span class="mi">8</span><span class="p">).</span><span class="na">equals</span><span class="p">(</span><span class="s">&quot;-&quot;</span><span class="p">))</span> <span class="p">{</span>
      <span class="n">fact</span><span class="p">.</span><span class="na">addDimensionValue</span><span class="p">(</span><span class="s">&quot;referrer&quot;</span><span class="p">,</span> <span class="n">matcher</span><span class="p">.</span><span class="na">group</span><span class="p">(</span><span class="mi">8</span><span class="p">));</span>
    <span class="p">}</span>
    <span class="n">fact</span><span class="p">.</span><span class="na">addDimensionValue</span><span class="p">(</span><span class="s">&quot;browser&quot;</span><span class="p">,</span> <span class="n">matcher</span><span class="p">.</span><span class="na">group</span><span class="p">(</span><span class="mi">9</span><span class="p">));</span>

    <span class="c1">// adding measurements</span>
    <span class="n">fact</span><span class="p">.</span><span class="na">addMeasurement</span><span class="p">(</span><span class="s">&quot;count&quot;</span><span class="p">,</span> <span class="n">MeasureType</span><span class="p">.</span><span class="na">COUNTER</span><span class="p">,</span> <span class="mi">1</span><span class="p">);</span>
    <span class="n">Integer</span> <span class="n">bytesSent</span> <span class="o">=</span> <span class="n">Integer</span><span class="p">.</span><span class="na">valueOf</span><span class="p">(</span><span class="n">matcher</span><span class="p">.</span><span class="na">group</span><span class="p">(</span><span class="mi">7</span><span class="p">));</span>
    <span class="n">fact</span><span class="p">.</span><span class="na">addMeasurement</span><span class="p">(</span><span class="s">&quot;bytes.sent&quot;</span><span class="p">,</span> <span class="n">MeasureType</span><span class="p">.</span><span class="na">COUNTER</span><span class="p">,</span> <span class="n">bytesSent</span><span class="p">);</span>
    <span class="n">cube</span><span class="p">.</span><span class="na">add</span><span class="p">(</span><span class="n">fact</span><span class="p">);</span>
  <span class="p">}</span>
<span class="p">}</span>
</pre></div>
</div>
<p>The <code class="docutils literal notranslate"><span class="pre">CubeWriterFlowlet</span></code> uses a Cube dataset that is injected via the <code class="docutils literal notranslate"><span class="pre">&#64;UseDataSet</span></code>
annotation with the specified dataset name. It reports on parsing errors by utilizing a
Metrics field injected by the CDAP framework.</p>
<p>The Flowlet process method parses the body of the <code class="docutils literal notranslate"><span class="pre">StreamEvent</span></code> that contains a log
entry in a combined log format. Then, it constructs a CubeFact by adding dimensions using the
parsed field values. It adds two measurements to be computed by the Cube in every
aggregation: the “count” for the number of requests, and the “bytes.sent” for the amount
of data sent.</p>
</div>
<div class="section" id="cubeservice">
<h4>CubeService<a class="headerlink" href="#cubeservice" title="Permalink to this headline">🔗</a></h4>
<p>The <code class="docutils literal notranslate"><span class="pre">CubeService</span></code> added to the Application is constructed using a single handler,
<code class="docutils literal notranslate"><span class="pre">CubeHandler</span></code>:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="kd">public</span> <span class="kd">final</span> <span class="kd">class</span> <span class="nc">CubeHandler</span> <span class="kd">extends</span> <span class="n">AbstractCubeHttpHandler</span> <span class="p">{</span>
  <span class="nd">@UseDataSet</span><span class="p">(</span><span class="n">WebAnalyticsApp</span><span class="p">.</span><span class="na">CUBE_NAME</span><span class="p">)</span>
  <span class="kd">private</span> <span class="n">Cube</span> <span class="n">cube</span><span class="p">;</span>

  <span class="nd">@Override</span>
  <span class="kd">protected</span> <span class="n">Cube</span> <span class="nf">getCube</span><span class="p">()</span> <span class="p">{</span>
    <span class="k">return</span> <span class="n">cube</span><span class="p">;</span>
  <span class="p">}</span>
<span class="p">}</span>
</pre></div>
</div>
<p>The <code class="docutils literal notranslate"><span class="pre">AbstractCubeHttpHandler</span></code> that is provided out-of-the-box with CDAP handles basic
Cube methods, such as <em>add</em>, <em>searchDimensionValue</em>, <em>searchMeasure</em>, and <em>query</em>, while the subclass
only needs to return the Cube dataset itself. Below, we will see how to use the HTTP
interface of the Service.</p>
</div>
</div>
</div>
<div class="section" id="build-and-run-application">
<h2>Build and Run Application<a class="headerlink" href="#build-and-run-application" title="Permalink to this headline">🔗</a></h2>
<p>The <code class="docutils literal notranslate"><span class="pre">WebAnalyticsApp</span></code> application can be built and packaged using the Apache Maven command:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$</span> mvn clean package
</pre></div>
</div>
<p>Note that the remaining commands assume that the <code class="docutils literal notranslate"><span class="pre">cdap-cli.sh</span></code> script is
available on your PATH. If that is not the case, please add it:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$</span> <span class="nb">export</span> <span class="nv">PATH</span><span class="o">=</span><span class="nv">$PATH</span>:&lt;CDAP home&gt;/bin
</pre></div>
</div>
<p>If you haven’t already started a standalone CDAP installation, start it with the command:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$</span> cdap.sh start
</pre></div>
</div>
<p>We can then deploy the application to a standalone CDAP installation and start <code class="docutils literal notranslate"><span class="pre">CubeWriterFlow</span></code>
and <code class="docutils literal notranslate"><span class="pre">CubeService</span></code>:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$</span> cdap-cli.sh load artifact target/cdap-cube-guide-&lt;version&gt;.jar
<span class="gp">$</span> cdap-cli.sh create app WebAnalyticsApp cdap-cube-guide &lt;version&gt; user
<span class="gp">$</span> cdap-cli.sh start flow WebAnalyticsApp.CubeWriterFlow
<span class="gp">$</span> cdap-cli.sh start service WebAnalyticsApp.CubeService
</pre></div>
</div>
<p>Next, we will send some sample weblogs into the Stream for processing:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$</span> cdap-cli.sh load stream weblogs resources/accesslog.txt
</pre></div>
</div>
<p>As data is being processed, we can start querying it via a RESTful API
provided by the <code class="docutils literal notranslate"><span class="pre">CubeService</span></code>. For convenience, we’ve put the queries themselves
into separate JSON files.</p>
<div class="section" id="explore-and-query-cube">
<h3>Explore and Query Cube<a class="headerlink" href="#explore-and-query-cube" title="Permalink to this headline">🔗</a></h3>
<p>Many times, users may not know what data a Cube contains and require some
exploration first to construct the queries themselves. Let’s start by searching
for the dimension values that are available in the Cube with this <code class="docutils literal notranslate"><span class="pre">CubeExploreQuery</span></code>:</p>
<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><span class="p">{</span>
    <span class="nt">&quot;startTs&quot;</span><span class="p">:</span> <span class="mi">1423370200</span><span class="p">,</span>
    <span class="nt">&quot;endTs&quot;</span><span class="p">:</span>   <span class="mi">1423398198</span><span class="p">,</span>
    <span class="nt">&quot;resolution&quot;</span><span class="p">:</span> <span class="mi">3600</span><span class="p">,</span>
    <span class="nt">&quot;dimensionValues&quot;</span><span class="p">:</span> <span class="p">[],</span>
    <span class="nt">&quot;limit&quot;</span><span class="p">:</span> <span class="mi">1000</span>
<span class="p">}</span>
</pre></div>
</div>
<p>Submit:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$</span> curl -w<span class="s1">&#39;\n&#39;</span> -X POST -d @resources/search-first.json <span class="s1">&#39;http://localhost:10000/v3/namespaces/default/apps/WebAnalyticsApp/services/CubeService/methods/searchDimensionValue&#39;</span>
</pre></div>
</div>
<p>The result will be the dimension values of the first dimensions defined in all aggregations (reformatted
for readability):</p>
<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><span class="p">[</span>
    <span class="p">{</span>
        <span class="nt">&quot;name&quot;</span><span class="p">:</span> <span class="s2">&quot;ip&quot;</span><span class="p">,</span>
        <span class="nt">&quot;value&quot;</span><span class="p">:</span> <span class="s2">&quot;69.181.160.120&quot;</span>
    <span class="p">},</span>
    <span class="p">{</span>
        <span class="nt">&quot;name&quot;</span><span class="p">:</span> <span class="s2">&quot;ip&quot;</span><span class="p">,</span>
        <span class="nt">&quot;value&quot;</span><span class="p">:</span> <span class="s2">&quot;109.63.206.34&quot;</span>
    <span class="p">},</span>
    <span class="p">{</span>
        <span class="nt">&quot;name&quot;</span><span class="p">:</span> <span class="s2">&quot;ip&quot;</span><span class="p">,</span>
        <span class="nt">&quot;value&quot;</span><span class="p">:</span> <span class="s2">&quot;113.72.144.115&quot;</span>
    <span class="p">},</span>
    <span class="p">{</span>
        <span class="nt">&quot;name&quot;</span><span class="p">:</span> <span class="s2">&quot;response_status&quot;</span><span class="p">,</span>
        <span class="nt">&quot;value&quot;</span><span class="p">:</span> <span class="s2">&quot;200&quot;</span>
    <span class="p">},</span>
    <span class="p">{</span>
        <span class="nt">&quot;name&quot;</span><span class="p">:</span> <span class="s2">&quot;response_status&quot;</span><span class="p">,</span>
        <span class="nt">&quot;value&quot;</span><span class="p">:</span> <span class="s2">&quot;404&quot;</span>
    <span class="p">}</span>
<span class="p">]</span>
</pre></div>
</div>
<p>To drill down further into the dimension hierarchy of aggregations, let’s refine the query with a specific dimension value:</p>
<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><span class="p">{</span>
    <span class="nt">&quot;startTs&quot;</span><span class="p">:</span> <span class="mi">1423370200</span><span class="p">,</span>
    <span class="nt">&quot;endTs&quot;</span><span class="p">:</span>   <span class="mi">1423398198</span><span class="p">,</span>
    <span class="nt">&quot;resolution&quot;</span><span class="p">:</span> <span class="mi">3600</span><span class="p">,</span>
    <span class="nt">&quot;dimensionValues&quot;</span><span class="p">:</span> <span class="p">[{</span><span class="nt">&quot;name&quot;</span><span class="p">:</span> <span class="s2">&quot;ip&quot;</span><span class="p">,</span> <span class="nt">&quot;value&quot;</span><span class="p">:</span> <span class="s2">&quot;69.181.160.120&quot;</span><span class="p">}],</span>
    <span class="nt">&quot;limit&quot;</span><span class="p">:</span> <span class="mi">1000</span>
<span class="p">}</span>
</pre></div>
</div>
<p>Submit:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$</span> curl -w<span class="s1">&#39;\n&#39;</span> -X POST -d @resources/search-ip.json <span class="s1">&#39;http://localhost:10000/v3/namespaces/default/apps/WebAnalyticsApp/services/CubeService/methods/searchDimensionValue&#39;</span>
</pre></div>
</div>
<p>The result is the dimension values of the next dimension defined in Cube aggregations:</p>
<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><span class="p">[</span>
    <span class="p">{</span>
        <span class="nt">&quot;name&quot;</span><span class="p">:</span> <span class="s2">&quot;browser&quot;</span><span class="p">,</span>
        <span class="nt">&quot;value&quot;</span><span class="p">:</span> <span class="s2">&quot;Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36&quot;</span>
    <span class="p">}</span>
<span class="p">]</span>
</pre></div>
</div>
<p>The Cube search API allows you to query for available measures via the <code class="docutils literal notranslate"><span class="pre">searchMeasure</span></code> endpoint:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$</span> curl -w<span class="s1">&#39;\n&#39;</span> -X POST -d @resources/search-ip.json <span class="s1">&#39;http://localhost:10000/v3/namespaces/default/apps/WebAnalyticsApp/services/CubeService/methods/searchMeasure&#39;</span>
</pre></div>
</div>
<p>The result contains all the measurement names:</p>
<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><span class="p">[</span>
    <span class="s2">&quot;bytes.sent&quot;</span><span class="p">,</span>
    <span class="s2">&quot;count&quot;</span>
<span class="p">]</span>
</pre></div>
</div>
<p>Now, let’s perform some data queries. Here’s how we can get the timeseries for the
number of bytes sent for a specific source ip, per each browser type:</p>
<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><span class="p">{</span>
    <span class="nt">&quot;aggregation&quot;</span><span class="p">:</span> <span class="s2">&quot;agg2&quot;</span><span class="p">,</span>
    <span class="nt">&quot;resolution&quot;</span><span class="p">:</span> <span class="mi">3600</span><span class="p">,</span>
    <span class="nt">&quot;startTs&quot;</span><span class="p">:</span> <span class="mi">1423370200</span><span class="p">,</span>
    <span class="nt">&quot;endTs&quot;</span><span class="p">:</span>   <span class="mi">1423398198</span><span class="p">,</span>
    <span class="nt">&quot;measurements&quot;</span><span class="p">:</span> <span class="p">{</span><span class="nt">&quot;bytes.sent&quot;</span><span class="p">:</span> <span class="s2">&quot;SUM&quot;</span><span class="p">},</span>
    <span class="nt">&quot;dimensionValues&quot;</span><span class="p">:</span> <span class="p">{</span><span class="nt">&quot;ip&quot;</span><span class="p">:</span> <span class="s2">&quot;69.181.160.120&quot;</span><span class="p">},</span>
    <span class="nt">&quot;groupByDimensions&quot;</span><span class="p">:</span> <span class="p">[</span><span class="s2">&quot;browser&quot;</span><span class="p">],</span>
    <span class="nt">&quot;limit&quot;</span><span class="p">:</span> <span class="mi">1000</span>
<span class="p">}</span>
</pre></div>
</div>
<p>One way of reading the query definition is this analogous SQL command:</p>
<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="k">SELECT</span>    <span class="k">sum</span><span class="p">(</span><span class="s1">&#39;bytes.sent&#39;</span><span class="p">)</span>                 <span class="c1">-- measure name and aggregation function</span>
<span class="k">FROM</span>      <span class="n">agg2</span><span class="p">.</span><span class="mi">1</span><span class="n">h_res</span>                       <span class="c1">-- aggregation &amp; resolution</span>
<span class="k">GROUP</span> <span class="k">BY</span>  <span class="n">browser</span>                           <span class="c1">-- groupByDimensions</span>
<span class="k">WHERE</span>     <span class="n">ip</span><span class="o">=</span><span class="s1">&#39;69.181.160.120&#39;</span> <span class="k">AND</span>           <span class="c1">-- dimensionValues</span>
          <span class="n">ts</span><span class="o">&gt;=</span><span class="mi">1423370200</span> <span class="k">AND</span> <span class="n">ts</span><span class="o">&lt;</span><span class="mi">1423398198</span>  <span class="c1">-- startTs &amp; endTs</span>
<span class="k">LIMIT</span>     <span class="mi">1000</span>                              <span class="c1">-- limit</span>
</pre></div>
</div>
<p>Submit:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$</span> curl -w<span class="s1">&#39;\n&#39;</span> -X POST -d @resources/query-ip-browser.json <span class="s1">&#39;http://localhost:10000/v3/namespaces/default/apps/WebAnalyticsApp/services/CubeService/methods/query&#39;</span>
</pre></div>
</div>
<p>The result is a timeseries with one data point (if any are available) per hour:</p>
<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><span class="p">[</span>
    <span class="p">{</span>
        <span class="nt">&quot;measureName&quot;</span><span class="p">:</span> <span class="s2">&quot;bytes.sent&quot;</span><span class="p">,</span>
        <span class="nt">&quot;dimensionValues&quot;</span><span class="p">:</span> <span class="p">{</span>
            <span class="nt">&quot;browser&quot;</span><span class="p">:</span> <span class="s2">&quot;Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36&quot;</span>
        <span class="p">},</span>
        <span class="nt">&quot;timeValues&quot;</span><span class="p">:</span> <span class="p">[</span>
            <span class="p">{</span>
                <span class="nt">&quot;timestamp&quot;</span><span class="p">:</span> <span class="mi">1423371600</span><span class="p">,</span>
                <span class="nt">&quot;value&quot;</span><span class="p">:</span> <span class="mi">122240</span>
            <span class="p">},</span>
            <span class="p">{</span>
                <span class="nt">&quot;timestamp&quot;</span><span class="p">:</span> <span class="mi">1423375200</span><span class="p">,</span>
                <span class="nt">&quot;value&quot;</span><span class="p">:</span> <span class="mi">122240</span>
            <span class="p">},</span>
            <span class="p">{</span>
                <span class="nt">&quot;timestamp&quot;</span><span class="p">:</span> <span class="mi">1423378800</span><span class="p">,</span>
                <span class="nt">&quot;value&quot;</span><span class="p">:</span> <span class="mi">121732</span>
            <span class="p">},</span>
            <span class="p">{</span>
                <span class="nt">&quot;timestamp&quot;</span><span class="p">:</span> <span class="mi">1423382400</span><span class="p">,</span>
                <span class="nt">&quot;value&quot;</span><span class="p">:</span> <span class="mi">122240</span>
            <span class="p">},</span>
            <span class="p">{</span>
                <span class="nt">&quot;timestamp&quot;</span><span class="p">:</span> <span class="mi">1423386000</span><span class="p">,</span>
                <span class="nt">&quot;value&quot;</span><span class="p">:</span> <span class="mi">121732</span>
            <span class="p">},</span>
            <span class="p">{</span>
                <span class="nt">&quot;timestamp&quot;</span><span class="p">:</span> <span class="mi">1423389600</span><span class="p">,</span>
                <span class="nt">&quot;value&quot;</span><span class="p">:</span> <span class="mi">122240</span>
            <span class="p">},</span>
            <span class="p">{</span>
                <span class="nt">&quot;timestamp&quot;</span><span class="p">:</span> <span class="mi">1423393200</span><span class="p">,</span>
                <span class="nt">&quot;value&quot;</span><span class="p">:</span> <span class="mi">121732</span>
            <span class="p">},</span>
            <span class="p">{</span>
                <span class="nt">&quot;timestamp&quot;</span><span class="p">:</span> <span class="mi">1423396800</span><span class="p">,</span>
                <span class="nt">&quot;value&quot;</span><span class="p">:</span> <span class="mi">47327</span>
            <span class="p">}</span>
        <span class="p">]</span>
    <span class="p">}</span>
<span class="p">]</span>
</pre></div>
</div>
<p>The query below will help to analyse the number of errors (or invalid requests) that the web site handles:</p>
<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><span class="p">{</span>
    <span class="nt">&quot;aggregation&quot;</span><span class="p">:</span> <span class="s2">&quot;agg1&quot;</span><span class="p">,</span>
    <span class="nt">&quot;startTs&quot;</span><span class="p">:</span> <span class="mi">1423370200</span><span class="p">,</span>
    <span class="nt">&quot;endTs&quot;</span><span class="p">:</span>   <span class="mi">1423398198</span><span class="p">,</span>
    <span class="nt">&quot;measurements&quot;</span><span class="p">:</span> <span class="p">{</span><span class="nt">&quot;count&quot;</span><span class="p">:</span> <span class="s2">&quot;SUM&quot;</span><span class="p">},</span>
    <span class="nt">&quot;resolution&quot;</span><span class="p">:</span> <span class="mi">3600</span><span class="p">,</span>
    <span class="nt">&quot;dimensionValues&quot;</span><span class="p">:</span> <span class="p">{},</span>
    <span class="nt">&quot;groupByDimensions&quot;</span><span class="p">:</span> <span class="p">[</span><span class="s2">&quot;response_status&quot;</span><span class="p">],</span>
    <span class="nt">&quot;limit&quot;</span><span class="p">:</span> <span class="mi">1000</span>
<span class="p">}</span>
</pre></div>
</div>
<p>Submit:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$</span> curl -w<span class="s1">&#39;\n&#39;</span> -X POST -d @resources/query-response-status.json <span class="s1">&#39;http://localhost:10000/v3/namespaces/default/apps/WebAnalyticsApp/services/CubeService/methods/query&#39;</span>
</pre></div>
</div>
<p>The result is a multiple timeseries for each response status:</p>
<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><span class="p">[</span>
    <span class="p">{</span>
        <span class="nt">&quot;measureName&quot;</span><span class="p">:</span> <span class="s2">&quot;count&quot;</span><span class="p">,</span>
        <span class="nt">&quot;dimensionValues&quot;</span><span class="p">:</span> <span class="p">{</span>
            <span class="nt">&quot;response_status&quot;</span><span class="p">:</span> <span class="s2">&quot;200&quot;</span>
        <span class="p">},</span>
        <span class="nt">&quot;timeValues&quot;</span><span class="p">:</span> <span class="p">[</span>
            <span class="p">{</span>
                <span class="nt">&quot;timestamp&quot;</span><span class="p">:</span> <span class="mi">1423371600</span><span class="p">,</span>
                <span class="nt">&quot;value&quot;</span><span class="p">:</span> <span class="mi">969</span>
            <span class="p">},</span>
            <span class="p">{</span>
                <span class="nt">&quot;timestamp&quot;</span><span class="p">:</span> <span class="mi">1423375200</span><span class="p">,</span>
                <span class="nt">&quot;value&quot;</span><span class="p">:</span> <span class="mi">360</span>
            <span class="p">},</span>
            <span class="p">{</span>
                <span class="nt">&quot;timestamp&quot;</span><span class="p">:</span> <span class="mi">1423378800</span><span class="p">,</span>
                <span class="nt">&quot;value&quot;</span><span class="p">:</span> <span class="mi">409</span>
            <span class="p">},</span>
            <span class="p">{</span>
                <span class="nt">&quot;timestamp&quot;</span><span class="p">:</span> <span class="mi">1423382400</span><span class="p">,</span>
                <span class="nt">&quot;value&quot;</span><span class="p">:</span> <span class="mi">468</span>
            <span class="p">},</span>
            <span class="p">{</span>
                <span class="nt">&quot;timestamp&quot;</span><span class="p">:</span> <span class="mi">1423386000</span><span class="p">,</span>
                <span class="nt">&quot;value&quot;</span><span class="p">:</span> <span class="mi">465</span>
            <span class="p">},</span>
            <span class="p">{</span>
                <span class="nt">&quot;timestamp&quot;</span><span class="p">:</span> <span class="mi">1423389600</span><span class="p">,</span>
                <span class="nt">&quot;value&quot;</span><span class="p">:</span> <span class="mi">468</span>
            <span class="p">},</span>
            <span class="p">{</span>
                <span class="nt">&quot;timestamp&quot;</span><span class="p">:</span> <span class="mi">1423393200</span><span class="p">,</span>
                <span class="nt">&quot;value&quot;</span><span class="p">:</span> <span class="mi">471</span>
            <span class="p">},</span>
            <span class="p">{</span>
                <span class="nt">&quot;timestamp&quot;</span><span class="p">:</span> <span class="mi">1423396800</span><span class="p">,</span>
                <span class="nt">&quot;value&quot;</span><span class="p">:</span> <span class="mi">186</span>
            <span class="p">}</span>
        <span class="p">]</span>
    <span class="p">},</span>
    <span class="p">{</span>
        <span class="nt">&quot;measureName&quot;</span><span class="p">:</span> <span class="s2">&quot;count&quot;</span><span class="p">,</span>
        <span class="nt">&quot;dimensionValues&quot;</span><span class="p">:</span> <span class="p">{</span>
            <span class="nt">&quot;response_status&quot;</span><span class="p">:</span> <span class="s2">&quot;404&quot;</span>
        <span class="p">},</span>
        <span class="nt">&quot;timeValues&quot;</span><span class="p">:</span> <span class="p">[</span>
            <span class="p">{</span>
                <span class="nt">&quot;timestamp&quot;</span><span class="p">:</span> <span class="mi">1423375200</span><span class="p">,</span>
                <span class="nt">&quot;value&quot;</span><span class="p">:</span> <span class="mi">2</span>
            <span class="p">},</span>
            <span class="p">{</span>
                <span class="nt">&quot;timestamp&quot;</span><span class="p">:</span> <span class="mi">1423378800</span><span class="p">,</span>
                <span class="nt">&quot;value&quot;</span><span class="p">:</span> <span class="mi">2</span>
            <span class="p">},</span>
            <span class="p">{</span>
                <span class="nt">&quot;timestamp&quot;</span><span class="p">:</span> <span class="mi">1423386000</span><span class="p">,</span>
                <span class="nt">&quot;value&quot;</span><span class="p">:</span> <span class="mi">2</span>
            <span class="p">},</span>
            <span class="p">{</span>
                <span class="nt">&quot;timestamp&quot;</span><span class="p">:</span> <span class="mi">1423393200</span><span class="p">,</span>
                <span class="nt">&quot;value&quot;</span><span class="p">:</span> <span class="mi">2</span>
            <span class="p">}</span>
        <span class="p">]</span>
    <span class="p">}</span>
<span class="p">]</span>
</pre></div>
</div>
<p>We can see there are just a few “404” responses, which is likely normal for such a
well-managed website(!).</p>
</div>
<div class="section" id="changing-the-cube-configuration">
<h3>Changing the Cube Configuration<a class="headerlink" href="#changing-the-cube-configuration" title="Permalink to this headline">🔗</a></h3>
<p>As applications evolve, we may need to change the Cube aggregation configuration to either
support new queries or to optimize existing ones. Let’s see how you can add an
aggregation to an existing Cube.</p>
<p>We’d like the configuration changed to include these properties:</p>
<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><span class="p">{</span>
    <span class="nt">&quot;dataset.cube.resolutions&quot;</span><span class="p">:</span><span class="s2">&quot;1,60,3600&quot;</span><span class="p">,</span>
    <span class="nt">&quot;dataset.cube.aggregation.agg1.dimensions&quot;</span><span class="p">:</span><span class="s2">&quot;response_status&quot;</span><span class="p">,</span>
    <span class="nt">&quot;dataset.cube.aggregation.agg2.dimensions&quot;</span><span class="p">:</span><span class="s2">&quot;ip,browser&quot;</span><span class="p">,</span>
    <span class="nt">&quot;dataset.cube.aggregation.agg3.dimensions&quot;</span><span class="p">:</span><span class="s2">&quot;referrer&quot;</span><span class="p">,</span>
    <span class="nt">&quot;dataset.cube.aggregation.agg3.requiredDimensions&quot;</span><span class="p">:</span><span class="s2">&quot;referrer&quot;</span>
<span class="p">}</span>
</pre></div>
</div>
<p>We’ve added <em>agg3</em> that computes statistics for referrers. Note the extra property that ends
with <em>requiredDimensions</em>: it specifies to only use this aggregation if the required dimension is present in a CubeFact.
You may have noticed that in <code class="docutils literal notranslate"><span class="pre">CubeWriterFlowlet</span></code>, the referrer field may be empty in a log entry.
We don’t want to store extra aggregates in the fact where this is the case.</p>
<p>Let’s update the dataset configuration, and then restart both the Flow and the Service so that the change takes effect:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$</span> curl -w<span class="s1">&#39;\n&#39;</span> -X PUT -d @resources/cube-config.json <span class="s1">&#39;http://localhost:10000/v3/namespaces/default/data/datasets/weblogsCube/properties&#39;</span>
<span class="gp">$</span> cdap-cli.sh stop flow WebAnalyticsApp.CubeWriterFlow
<span class="gp">$</span> cdap-cli.sh start flow WebAnalyticsApp.CubeWriterFlow
<span class="gp">$</span> cdap-cli.sh stop service WebAnalyticsApp.CubeService
<span class="gp">$</span> cdap-cli.sh start service WebAnalyticsApp.CubeService
</pre></div>
</div>
<p>Let’s send additional data to compute new aggregations:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$</span> cdap-cli.sh load stream weblogs resources/accesslog.txt
</pre></div>
</div>
<p>Now, we can retrieve statistics on referrers using the newly-added aggregation:</p>
<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><span class="p">{</span>
    <span class="nt">&quot;aggregation&quot;</span><span class="p">:</span> <span class="s2">&quot;agg3&quot;</span><span class="p">,</span>
    <span class="nt">&quot;startTs&quot;</span><span class="p">:</span> <span class="mi">1423370200</span><span class="p">,</span>
    <span class="nt">&quot;endTs&quot;</span><span class="p">:</span>   <span class="mi">1423398198</span><span class="p">,</span>
    <span class="nt">&quot;measurements&quot;</span><span class="p">:</span> <span class="p">{</span><span class="nt">&quot;count&quot;</span><span class="p">:</span> <span class="s2">&quot;SUM&quot;</span><span class="p">},</span>
    <span class="nt">&quot;resolution&quot;</span><span class="p">:</span> <span class="mi">3600</span><span class="p">,</span>
    <span class="nt">&quot;dimensionValues&quot;</span><span class="p">:</span> <span class="p">{</span><span class="nt">&quot;referrer&quot;</span><span class="p">:</span> <span class="s2">&quot;http://cdap.io/&quot;</span><span class="p">},</span>
    <span class="nt">&quot;groupByDimensions&quot;</span><span class="p">:</span> <span class="p">[],</span>
    <span class="nt">&quot;limit&quot;</span><span class="p">:</span> <span class="mi">1000</span>
<span class="p">}</span>
</pre></div>
</div>
<p>Submit:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$</span> curl -w<span class="s1">&#39;\n&#39;</span> -X POST -d @resources/query-referrer.json <span class="s1">&#39;http://localhost:10000/v3/namespaces/default/apps/WebAnalyticsApp/services/CubeService/methods/query&#39;</span>
</pre></div>
</div>
<p>Result:</p>
<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><span class="p">[</span>
    <span class="p">{</span>
        <span class="nt">&quot;measureName&quot;</span><span class="p">:</span> <span class="s2">&quot;count&quot;</span><span class="p">,</span>
        <span class="nt">&quot;dimensionValues&quot;</span><span class="p">:</span> <span class="p">{},</span>
        <span class="nt">&quot;timeValues&quot;</span><span class="p">:</span> <span class="p">[</span>
            <span class="p">{</span>
                <span class="nt">&quot;timestamp&quot;</span><span class="p">:</span> <span class="mi">1423375200</span><span class="p">,</span>
                <span class="nt">&quot;value&quot;</span><span class="p">:</span> <span class="mi">3</span>
            <span class="p">},</span>
            <span class="p">{</span>
                <span class="nt">&quot;timestamp&quot;</span><span class="p">:</span> <span class="mi">1423389600</span><span class="p">,</span>
                <span class="nt">&quot;value&quot;</span><span class="p">:</span> <span class="mi">1</span>
            <span class="p">}</span>
        <span class="p">]</span>
    <span class="p">}</span>
<span class="p">]</span>
</pre></div>
</div>
</div>
</div>
<div class="section" id="share-and-discuss">
<h2>Share and Discuss!<a class="headerlink" href="#share-and-discuss" title="Permalink to this headline">🔗</a></h2>
<p>Have a question? Discuss at the <a class="reference external" href="https://groups.google.com/forum/#!forum/cdap-user">CDAP User Mailing List</a>.</p>
</div>
<div class="section" id="license">
<h2>License<a class="headerlink" href="#license" title="Permalink to this headline">🔗</a></h2>
<p>Copyright © 2015 Cask Data, Inc.</p>
<p>Licensed under the Apache License, Version 2.0 (the “License”); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at</p>
<p><a class="reference external" href="http://www.apache.org/licenses/LICENSE-2.0">http://www.apache.org/licenses/LICENSE-2.0</a></p>
<p>Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an “AS IS” BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.</p>
</div>
</div>

</div>
    <div class="col-md-2">
      <div id="right-sidebar" class="bs-sidenav scrollable-y" role="complementary">
        <div id="localtoc-scrollspy">
        </div>
      </div>
    </div></div>
</div>
<!-- block main content end -->
<!-- block footer -->
<footer class="footer">
      <div class="container">
        <div class="row">
          <div class="col-md-2 footer-left"><a title="Analyzing CDAP Data from BI Tools" href="cdap-bi-guide.html" />Previous</a></div>
          <div class="col-md-8 footer-center"><a class="footer-tab-link" href="../table-of-contents/../../reference-manual/licenses/index.html">Copyright</a> &copy; 2014-2020 Cask Data, Inc.&bull; <a class="footer-tab-link" href="//docs.cask.co/cdap/6.1.1/cdap-docs-6.1.1-web.zip" rel="nofollow">Download</a> an archive or
<a class="footer-tab-link" href="//docs.cask.co/cdap">switch the version</a> of the documentation
          </div>
          <div class="col-md-2 footer-right"><a title="Creating Hydrator Applications using CDAP System Artifacts" href="cdap-etl-guide.html" />Next</a></div>
        </div>
      </div>
    </footer>
<!-- block footer end -->
<script type="text/javascript" src="../_static/bootstrap-3.3.6/js/bootstrap.min.js"></script><script type="text/javascript" src="../_static/js/bootstrap-sphinx.js"></script><script type="text/javascript" src="../_static/js/abixTreeList-2.js"></script><script type="text/javascript" src="../_static/js/cdap-dynamicscrollspy-4.js"></script><script type="text/javascript" src="../_static/js/cdap-version-menu.js"></script><script type="text/javascript" src="../_static/js/copy-to-clipboard.js"></script><script type="text/javascript" src="../_static/js/jquery.mousewheel.min.js"></script><script type="text/javascript" src="../_static/js/jquery.mCustomScrollbar.js"></script><script type="text/javascript" src="../_static/js/js.cookie.js"></script><script type="text/javascript" src="../_static/js/tabbed-parsed-literal-0.2.js"></script><script type="text/javascript" src="../_static/js/cdap-onload-javascript.js"></script><script type="text/javascript" src="../_static/js/cdap-version-menu.js"></script>
    <script src="https://cdap.gitee.io/docs/cdap/json-versions.js"/></script>
  </body>
</html>